import csv
import time

import jsonpath
import requests


# 创建session对象
# s = requests.session()
class SpiderRealEstate(object):
    ip = "221.9.131.206"
    port = 4323

    def __init__(self):
        self.headers = {
            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.99 Safari/537.36"
        }
        self.proxies = {
            "http": "http://%s:%d" % (SpiderRealEstate.ip, SpiderRealEstate.port),
            "https": "http://%s:%d" % (SpiderRealEstate.ip, SpiderRealEstate.port)
        }
        self.page = 1
        self.list1 = list()

    def spider(self):
        for i in range(5):
            url = "https://shuyang.anjuke.com/v3/ajax/map/sale/1338/prop_list/?kw=%E6%B2%AD%E9%98%B3&room_num=-1&price_id=-1&area_id=-1&floor=-1&orientation=-1&is_two_years=0&is_school=0&is_metro=0&order_id=0&p={}&zoom=12&maxp=99&et=c8b35e&ib=1&bst=pem834".format(
                self.page)
            # r1 = s.get(url=url, headers=headers, proxies=proxies)
            r = requests.get(url=url, headers=self.headers, verify=False, timeout=10)
            time.sleep(5)

            rooms = r.json()
            ret = jsonpath.jsonpath(rooms, "$..props")

            for item in ret[0]:
                dict1 = dict()
                region_name = jsonpath.jsonpath(item, "$.region_name")
                dict1["region_name"] = region_name[0]
                block_name = jsonpath.jsonpath(item, "$.block_name")
                dict1["block_name"] = block_name[0]
                rhval = jsonpath.jsonpath(item, "$.rhval")
                dict1["rhval"] = rhval[0]
                area = jsonpath.jsonpath(item, "$.area")
                dict1["area"] = area[0]
                price = jsonpath.jsonpath(item, "$.price")
                dict1["price"] = price[0]
                comm_id = jsonpath.jsonpath(item, "$.comm_id")
                dict1["comm_id"] = comm_id[0]
                long_title = jsonpath.jsonpath(item, "$.long_title")
                dict1["long_title"] = long_title[0]
                comm_name = jsonpath.jsonpath(item, "$.comm_name")
                dict1["comm_name"] = comm_name[0]
                house_orient_name = jsonpath.jsonpath(item, "$.house_orient_name")
                dict1["house_orient_name"] = house_orient_name[0]
                floor_tag = jsonpath.jsonpath(item, "$.floor_tag")
                dict1["floor_tag"] = floor_tag[0]
                self.list1.append(dict1)
            self.page += 1

    def save(self):
        with open('沭阳房产-20201106.csv', 'w', encoding="utf-8") as f:
            # 创建一个csv的DictWriter对象，这样才能够将写入csv格式数据到这个文件
            f_csv = csv.DictWriter(f,
                                   ["region_name", "block_name", "rhval", "area", "price", "comm_id", "long_title",
                                    "comm_name",
                                    "house_orient_name", "floor_tag"])
            # 写入一行（我们用第一行当做表头）
            f_csv.writeheader()
            # 写入多行行（当做数据）
            f_csv.writerows(self.list1)


spider = SpiderRealEstate()
spider.spider()
spider.save()

# # http代理
# ip = "221.9.131.206"
# port = 4323
#
# proxies = {
#     "http": "http://%s:%d" % (ip, port),
#     "https": "http://%s:%d" % (ip, port)
# }
#
# # 请求头
# headers = {
#     "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.99 Safari/537.36"
# }
#
# # url = "https://shuyang.anjuke.com/map/sale/"
# page = 1
# list1 = list()
# for i in range(5):
#     url = "https://shuyang.anjuke.com/v3/ajax/map/sale/1338/prop_list/?kw=%E6%B2%AD%E9%98%B3&room_num=-1&price_id=-1&area_id=-1&floor=-1&orientation=-1&is_two_years=0&is_school=0&is_metro=0&order_id=0&p={}&zoom=12&maxp=99&et=c8b35e&ib=1&bst=pem834".format(
#         page)
#     # r1 = s.get(url=url, headers=headers, proxies=proxies)
#     r = requests.get(url=url, headers=headers)
#     time.sleep(5)
#
#     rooms = r.json()
#     ret = jsonpath.jsonpath(rooms, "$..props")
#
#     for item in ret[0]:
#         dict1 = dict()
#         region_name = jsonpath.jsonpath(item, "$.region_name")
#         dict1["region_name"] = region_name[0]
#         block_name = jsonpath.jsonpath(item, "$.block_name")
#         dict1["block_name"] = block_name[0]
#         rhval = jsonpath.jsonpath(item, "$.rhval")
#         dict1["rhval"] = rhval[0]
#         area = jsonpath.jsonpath(item, "$.area")
#         dict1["area"] = area[0]
#         price = jsonpath.jsonpath(item, "$.price")
#         dict1["price"] = price[0]
#         comm_id = jsonpath.jsonpath(item, "$.comm_id")
#         dict1["comm_id"] = comm_id[0]
#         long_title = jsonpath.jsonpath(item, "$.long_title")
#         dict1["long_title"] = long_title[0]
#         comm_name = jsonpath.jsonpath(item, "$.comm_name")
#         dict1["comm_name"] = comm_name[0]
#         house_orient_name = jsonpath.jsonpath(item, "$.house_orient_name")
#         dict1["house_orient_name"] = house_orient_name[0]
#         floor_tag = jsonpath.jsonpath(item, "$.floor_tag")
#         dict1["floor_tag"] = floor_tag[0]
#         list1.append(dict1)
#     page += 1
# with open('沭阳房产-20201106.csv', 'w', encoding="utf-8") as f:
#     # 创建一个csv的DictWriter对象，这样才能够将写入csv格式数据到这个文件
#     f_csv = csv.DictWriter(f,
#                            ["region_name", "block_name", "rhval", "area", "price", "comm_id", "long_title", "comm_name",
#                             "house_orient_name", "floor_tag"])
#     # 写入一行（我们用第一行当做表头）
#     f_csv.writeheader()
#     # 写入多行行（当做数据）
#     f_csv.writerows(list1)
